{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "\n",
    "from __future__ import print_function"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# XOR Network"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Data generation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def create_examples(N, batch_size):\n",
    "    A = np.random.binomial(n=1, p=0.5, size=(batch_size, N))\n",
    "    B = np.random.binomial(n=1, p=0.5, size=(batch_size, N,))\n",
    "\n",
    "    X = np.zeros((batch_size, 2 *N,), dtype=np.float32)\n",
    "    X[:,:N], X[:,N:] = A, B\n",
    "\n",
    "    Y = (A ^ B).astype(np.float32)\n",
    "    return X,Y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ 0.  1.  0.] xor [ 1.  1.  1.] equals [ 1.  0.  1.]\n",
      "[ 0.  0.  1.] xor [ 1.  1.  0.] equals [ 1.  1.  1.]\n"
     ]
    }
   ],
   "source": [
    "X, Y = create_examples(3, 2)\n",
    "print(X[0,:3], \"xor\", X[0,3:],\"equals\", Y[0])\n",
    "print(X[1,:3], \"xor\", X[1,3:],\"equals\", Y[1])\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Xor cannot be solved with single layer of neural network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import math\n",
    "\n",
    "class Layer(object):\n",
    "    def __init__(self, input_size, output_size):\n",
    "        tensor_b = tf.zeros((output_size,))\n",
    "        self.b = tf.Variable(tensor_b)\n",
    "        tensor_W = tf.random_uniform((input_size, output_size),\n",
    "                                     -1.0 / math.sqrt(input_size),\n",
    "                                     1.0 / math.sqrt(input_size))\n",
    "        self.W = tf.Variable(tensor_W)\n",
    "\n",
    "    def __call__(self, x):\n",
    "        return tf.matmul(x, self.W) + self.b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "tf.ops.reset_default_graph()\n",
    "sess = tf.InteractiveSession()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "N = 5\n",
    "# x represents input data\n",
    "x = tf.placeholder(tf.float32, (None, 2 * N), name=\"x\")\n",
    "# y_golden is a reference output data.\n",
    "y_golden = tf.placeholder(tf.float32, (None, N), name=\"y\")\n",
    "\n",
    "layer1 = Layer(2 * N, N)\n",
    "# y is a linear projection of x with nonlinearity applied to the result.\n",
    "y = tf.nn.sigmoid(layer1(x))\n",
    "\n",
    "# mean squared error over all examples and all N output dimensions.\n",
    "cost = tf.reduce_mean(tf.square(y - y_golden))\n",
    "\n",
    "# create a function that will optimize the neural network\n",
    "optimizer = tf.train.AdagradOptimizer(learning_rate=0.3)\n",
    "train_op = optimizer.minimize(cost)\n",
    "\n",
    "# initialize the variables\n",
    "sess.run(tf.initialize_all_variables())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.262958\n",
      "0.249229\n",
      "0.259427\n",
      "0.245061\n",
      "0.252946\n",
      "0.24782\n",
      "0.250937\n",
      "0.246418\n",
      "0.246755\n",
      "0.244774\n"
     ]
    }
   ],
   "source": [
    "for t in range(5000):\n",
    "    example_x, example_y = create_examples(N, 10)\n",
    "    cost_t, _ = sess.run([cost, train_op], {x: example_x, y_golden: example_y})\n",
    "    if t % 500 == 0: \n",
    "        print(cost_t.mean())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Notice that the error is far from zero.\n",
    "\n",
    "Actually network always predicts approximately $0.5$, regardless of input data. That yields error of about $0.25$, because we use mean squared error ($0.5^2 = 0.25$). "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 1.  0.  1.  1.  1.  1.  0.  0.  1.  1.]\n",
      " [ 1.  0.  1.  1.  0.  1.  1.  1.  1.  1.]\n",
      " [ 0.  0.  1.  0.  1.  0.  0.  1.  1.  1.]]\n",
      "[array([[ 0.56099683,  0.54470569,  0.4940519 ,  0.49518651,  0.54470527],\n",
      "       [ 0.56658453,  0.52068532,  0.48442408,  0.4748241 ,  0.5073036 ],\n",
      "       [ 0.53004831,  0.52866411,  0.48705727,  0.48926324,  0.53761232]], dtype=float32)]\n"
     ]
    }
   ],
   "source": [
    "X, _ = create_examples(N, 3)\n",
    "prediction = sess.run([y], {x: X})\n",
    "print(X)\n",
    "print(prediction)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Accuracy is not that hard to predict..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy over 1000 examples: 48 %\n"
     ]
    }
   ],
   "source": [
    "N_EXAMPLES = 1000\n",
    "example_x, example_y = create_examples(N, N_EXAMPLES)\n",
    "# one day I need to write a wrapper which will turn the expression\n",
    "# below to:\n",
    "#     tf.abs(y - y_golden) < 0.5\n",
    "is_correct = tf.less_equal(tf.abs(y - y_golden), tf.constant(0.5))\n",
    "accuracy = tf.reduce_mean(tf.cast(is_correct, \"float\"))\n",
    "\n",
    "acc_result = sess.run(accuracy, {x: example_x, y_golden: example_y})\n",
    "print(\"Accuracy over %d examples: %.0f %%\" % (N_EXAMPLES, 100.0 * acc_result))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Xor Network with 2 layers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Exception AssertionError: AssertionError() in <bound method InteractiveSession.__del__ of <tensorflow.python.client.session.InteractiveSession object at 0x7f56d57e8dd0>> ignored\n"
     ]
    }
   ],
   "source": [
    "tf.ops.reset_default_graph()\n",
    "sess = tf.InteractiveSession()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "N = 5\n",
    "# we add a single hidden layer of size 12\n",
    "# otherwise code is similar to above\n",
    "HIDDEN_SIZE = 12\n",
    "\n",
    "x = tf.placeholder(tf.float32, (None, 2 * N), name=\"x\")\n",
    "y_golden = tf.placeholder(tf.float32, (None, N), name=\"y\")\n",
    "\n",
    "layer1 = Layer(2 * N, HIDDEN_SIZE)\n",
    "layer2 = Layer(HIDDEN_SIZE, N) # <------- HERE IT IS!\n",
    "\n",
    "hidden_repr = tf.nn.tanh(layer1(x))\n",
    "y = tf.nn.sigmoid(layer2(hidden_repr))\n",
    "\n",
    "cost = tf.reduce_mean(tf.square(y - y_golden))\n",
    "\n",
    "optimizer = tf.train.AdagradOptimizer(learning_rate=0.3)\n",
    "train_op = optimizer.minimize(cost)\n",
    "sess.run(tf.initialize_all_variables())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.241089\n",
      "0.240045\n",
      "0.1631\n",
      "0.0709099\n",
      "0.0326128\n",
      "0.0087687\n",
      "0.00526247\n",
      "0.00518266\n",
      "0.00272845\n",
      "0.00213744\n"
     ]
    }
   ],
   "source": [
    "for t in range(5000):\n",
    "    example_x, example_y = create_examples(N, 10)\n",
    "    cost_t, _ = sess.run([cost, train_op], {x: example_x, y_golden: example_y})\n",
    "    if t % 500 == 0: \n",
    "        print(cost_t.mean())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### This time the network works a tad better"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 0.  0.  0.  0.  0.  0.  0.  0.  0.  1.]\n",
      " [ 1.  0.  0.  1.  1.  1.  0.  1.  1.  1.]\n",
      " [ 0.  1.  1.  1.  0.  0.  0.  0.  1.  0.]]\n",
      "[[ 0.  0.  0.  0.  1.]\n",
      " [ 0.  0.  1.  0.  0.]\n",
      " [ 0.  1.  1.  0.  0.]]\n",
      "[array([[ 0.10384335,  0.04389301,  0.05774897,  0.04509954,  0.9374879 ],\n",
      "       [ 0.05130127,  0.02655722,  0.97246277,  0.03545236,  0.04168396],\n",
      "       [ 0.03924223,  0.96327722,  0.96935028,  0.03265698,  0.0310236 ]], dtype=float32)]\n"
     ]
    }
   ],
   "source": [
    "X, Y = create_examples(N, 3)\n",
    "prediction = sess.run([y], {x: X})\n",
    "print(X)\n",
    "print(Y)\n",
    "print(prediction)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy over 1000 examples: 100 %\n"
     ]
    }
   ],
   "source": [
    "N_EXAMPLES = 1000\n",
    "example_x, example_y = create_examples(N, N_EXAMPLES)\n",
    "is_correct = tf.less_equal(tf.abs(y - y_golden), tf.constant(0.5))\n",
    "accuracy = tf.reduce_mean(tf.cast(is_correct, \"float\"))\n",
    "\n",
    "acc_result = sess.run(accuracy, {x: example_x, y_golden: example_y})\n",
    "print(\"Accuracy over %d examples: %.0f %%\" % (N_EXAMPLES, 100.0 * acc_result))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}
